﻿using Newtonsoft.Json.Linq;
using Reborn.Infrastructure;
using SimpleSpider.Framework.Downloader;
using SimpleSpider.Framework.Model;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;


namespace SimpleSpider.Framework
{
    public class EasyCrawlUrlWorker : CrawlUrlWorker
    {
        IDownloader _downloader = null;

        public EasyCrawlUrlWorker()
        {
            _downloader = new HttpDownloader();
            _downloader.SetRequest("User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36\r\nAccept: text/html,application/xhtml+");

        }

        public override void Handler(StartUrlModel startUrlModel)
        {
            int numberOfduplicates = 0;
            string oldHtml = string.Empty;

            for (int i = 1; i < 50; i++)
            {


                string url = startUrlModel.Url.Replace("{分页}", i.ToString());
                string shtml = _downloader.Get(url);

                if (oldHtml == shtml)
                {
                    numberOfduplicates++;
                    if (numberOfduplicates > 3)
                    {
                        //重复了
                        break;
                    }
                    continue;
                }
                numberOfduplicates = 0;
                oldHtml = shtml;

                JToken dataResult = JObject.Parse(shtml);
                foreach (var item in dataResult["data"]["results"])
                {
                    try
                    {
                        string id = item["id"].ToString();
                        string mUrl = item["mUrl"].ToString();

                        if (string.IsNullOrEmpty(mUrl)) mUrl = id;

                        if (IsUnique(id))
                        {
                            continue;
                        }

                        numberOfduplicates = 0;

                        UrlModel urlModel = new UrlModel();
                        urlModel.Url = mUrl;
                        urlModel.UniqueId = id;
                        urlModel.StartUrlId = startUrlModel.Id;

                        Save(urlModel);
                    }
                    catch (Exception ex)
                    {
                        Logger.Exception(ex);
                    }

                }
                int numner = new Random().Next(5, 10);
                Thread.Sleep(numner * 1000);
            }


            startUrlModel.Status = Reborn.Contracts.State.TaskStatus.RanToCompletion;
        }

    }
}
